package com.tingfeng.ik;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;

import java.io.IOException;

/**
 * IK 中文分词 测试
 */
public class IkChineseAnalyzerDemo {
    private static void doToken(TokenStream ts) throws IOException {
        ts.reset();
        CharTermAttribute cta = ts.getAttribute(CharTermAttribute.class);
        while (ts.incrementToken()) {
            System.out.print(cta.toString() + "|");
        }
        System.out.println();
        System.out.println();
        ts.end();
        ts.close();
    }

    public static void main(String[] args) {
        String etext = "Analysis is one of the main causes of slow indexing. Simply put, ";
        String chineseText = "厉害了我的国一经播出，受到各方好评，强烈激发了国人的爱国之情、自豪感，Television 电视机转播！";
        try {
            // IKAnalyzer 细粒度切分
            Analyzer ik = new IKAnalyzer4Lucene7();
            TokenStream ts = ik.tokenStream("content", etext);
            System.out.println("IKAnalyzer中文分词器 细粒度切分，英文分词效果：");
            doToken(ts);
            ts = ik.tokenStream("content", chineseText);
            System.out.println("IKAnalyzer中文分词器 细粒度切分，中文分词效果：");
            doToken(ts);
        } catch (Exception e) {
            e.printStackTrace();
        }

        try {
            // IKAnalyzer 智能切分
            Analyzer ik = new IKAnalyzer4Lucene7(true);
            TokenStream ts = ik.tokenStream("content", etext);
            System.out.println("IKAnalyzer中文分词器 智能切分，英文分词效果：");
            doToken(ts);
            ts = ik.tokenStream("content", chineseText);
            System.out.println("IKAnalyzer中文分词器 智能切分，中文分词效果：");
            doToken(ts);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

}
